package com.chance.cc.crawler.development.scripts.douyin;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.KVTag;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.*;

/**
 * @author bx
 * @date 2020/12/6 0006 12:52
 */
public class DYCommonCrawlerScript extends CrawlerCommonScript {

    private static Logger logger = LoggerFactory.getLogger(DYCommonCrawlerScript.class);

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        return null;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (requestUrl.contains("aweme/v1/general/search/single")){ //search video
            return washSearchVideo(crawlerRecord,page);
        }
        if (requestUrl.contains("aweme/v1/aweme/post")){ //search video
            return washAccountVideo(crawlerRecord,page);
        }
        if (requestUrl.contains("aweme/v2/comment/list")){ //search video
            return washVideoComment(crawlerRecord,page);
        }
        return null;
    }

    private List<CrawlerData> washSearchVideo(CrawlerRequestRecord crawlerRecord, HttpPage page){
        HttpPage dy_search_page = crawlerRecord.getInternalDownloadPage();
        List<CrawlerData> crawlerDataList = new ArrayList<>();

        KVTag dyRequestData = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getKVTag("dy_request_data");
        List<NameValuePair> parameters = URLEncodedUtils.parse((String) dyRequestData.getVal(), Charset.forName("utf-8"));
        String keyword = "";
        for (NameValuePair parameter : parameters) {
            if ("keyword".equals(parameter.getName())){
                keyword = parameter.getValue();
                break;
            }
        }

        Json searchJson = new Json(dy_search_page.getRawText());
        List<String> datas = searchJson.jsonPath($_type + ".data").all();
        if(datas != null){
            for(String data : datas){
                try {
                    Json dataJson = new Json(data);
                    if ("1".equals(dataJson.jsonPath($_type + ".type").get())){
                        String aweme_id = dataJson.jsonPath($_type + ".aweme_info.aweme_id").get();
                        String create_time = dataJson.jsonPath($_type + ".aweme_info.create_time").get();
                        CrawlerData crawlerData = CrawlerData.builder()
                                .data(crawlerRecord, page)
                                .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), aweme_id))
                                .url("")
                                .releaseTime(Long.valueOf(create_time ) * 1000L)
                                .addContentKV(Field_Title,dataJson.jsonPath($_type + ".aweme_info.desc").get())
                                .addContentKV(Field_Author,dataJson.jsonPath($_type + ".aweme_info.author.nickname").get())
                                .addContentKV(Field_Author_Id,dataJson.jsonPath($_type + ".aweme_info.author.uid").get())
                                .addContentKV(Field_I_Forwards,dataJson.jsonPath($_type + ".aweme_info.statistics.share_count").get())
                                .addContentKV(Field_I_Comments,dataJson.jsonPath($_type + ".aweme_info.statistics.comment_count").get())
                                .addContentKV(Field_I_Likes,dataJson.jsonPath($_type + ".aweme_info.statistics.digg_count").get())
                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .build();
                        crawlerData.tagsCreator().bizTags().addCustomKV(Tag_Field_Keword,keyword);
                        crawlerDataList.add(crawlerData);
                    }
                } catch (Exception e) {
                    logger.error("dy search video content data normal,error: {}",e.getMessage());
                }
            }
        }
        return crawlerDataList;
    }

    private List<CrawlerData> washAccountVideo(CrawlerRequestRecord crawlerRecord, HttpPage page){
        HttpPage dy_account_page = crawlerRecord.getInternalDownloadPage();
        List<CrawlerData> crawlerDataList = new ArrayList<>();

        Json searchJson = new Json(dy_account_page.getRawText());
        List<String> datas = searchJson.jsonPath($_type + ".aweme_list").all();
        if (datas != null){
            for(String data : datas){
                try {
                    Json dataJson = new Json(data);
                    String aweme_id = dataJson.jsonPath($_type + ".aweme_id").get();
                    String create_time = dataJson.jsonPath($_type + ".create_time").get();
                    String uniqueId = dataJson.jsonPath(".author.unique_id").get();
                    if (StringUtils.isBlank(uniqueId)){
                        uniqueId = dataJson.jsonPath(".author.short_id").get();
                    }
                    CrawlerData crawlerData = CrawlerData.builder()
                            .data(crawlerRecord, page)
                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), aweme_id))
                            .url("")
                            .releaseTime(Long.valueOf(create_time ) * 1000L)
                            .addContentKV(Field_Title,dataJson.jsonPath($_type + ".desc").get())
                            .addContentKV(Field_Author,dataJson.jsonPath($_type + ".author.nickname").get())
                            .addContentKV(Field_Author_Id,dataJson.jsonPath($_type + ".author.uid").get())
                            .addContentKV(Field_I_Forwards,dataJson.jsonPath($_type + ".statistics.share_count").get())
                            .addContentKV(Field_I_Comments,dataJson.jsonPath($_type + ".statistics.comment_count").get())
                            .addContentKV(Field_I_Likes,dataJson.jsonPath($_type + ".statistics.digg_count").get())
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .build();
                    crawlerData.tagsCreator().bizTags().addCustomKV(Tag_Field_Keword,uniqueId);

                    crawlerDataList.add(crawlerData);
                } catch (Exception e) {
                    logger.error("dy account video content data normal,error: {}",e.getMessage());
                }
            }
        }
        return crawlerDataList;
    }

    private List<CrawlerData> washVideoComment(CrawlerRequestRecord crawlerRecord, HttpPage page){
        HttpPage dy_account_page = crawlerRecord.getInternalDownloadPage();
        List<CrawlerData> crawlerDataList = new ArrayList<>();

        Json searchJson = new Json(dy_account_page.getRawText());
        List<String> comments = searchJson.jsonPath($_type + ".comments").all();
        if (comments != null){
            for(String comment : comments){
                try {
                    Json dataJson = new Json(comment);
                    String cid = dataJson.jsonPath($_type + ".cid").get();
                    String aweme_id = dataJson.jsonPath($_type + ".aweme_id").get();
                    String create_time = dataJson.jsonPath($_type + ".create_time").get();

                    CrawlerData crawlerData = CrawlerData.builder()
                            .data(crawlerRecord, page)
                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.comment.enumVal(), cid))
                            .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), aweme_id))
                            .url("")
                            .releaseTime(Long.valueOf(create_time ) * 1000L)
                            .addContentKV(Field_Author,dataJson.jsonPath($_type + ".user.nickname").get())
                            .addContentKV(Field_Author_Id,dataJson.jsonPath($_type + ".user.uid").get())
                            .addContentKV(Field_Content,dataJson.jsonPath($_type + ".text").get())
                            .addContentKV(Field_I_Comments,dataJson.jsonPath($_type + ".reply_comment_total").get())
                            .addContentKV(Field_I_Likes,dataJson.jsonPath($_type + ".digg_count").get())
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                            .build();

                    crawlerDataList.add(crawlerData);
                } catch (Exception e) {
                    logger.error("dy account video content data normal,error: {}",e.getMessage());
                }
            }
        }
        return crawlerDataList;
    }

    @Override
    public String domain() {
        return "douyin";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("/aweme/v1/crawler/start/url"); //start url--只是为了进入脚本没有实际意义
        addUrlRegular("https://\\S*/aweme/v1/general/search/single\\S*"); //查询接口
        addUrlRegular("https://\\S*/aweme/v1/aweme/post\\S*"); //用户接口
        addUrlRegular("https://\\S*/aweme/v2/comment/list\\S*"); //评论接口
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {

        HttpPage httpPage = crawlerRecord.getInternalDownloadPage();
        if (httpPage == null){
            logger.error("current crawler record [{}] has not dy content!", JSON.toJSONString(crawlerRecord));
            return false;
        }
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        String requestUrl = requestRecord.getHttpRequest().getUrl();
        if (requestUrl.contains("/aweme/v1/general/search/single")){ //search video
            fillCurrentRecord(requestRecord);
        }
        if (requestUrl.contains("/aweme/v1/aweme/post")){ //search video
            fillCurrentRecord(requestRecord);
        }
        if (requestUrl.contains("/aweme/v2/comment/list")){ //search video
            fillCurrentRecord(requestRecord);
        }
        return super.prepareRequest(requestRecord, supportSourceRecords);
    }

    private void fillCurrentRecord(CrawlerRequestRecord requestRecord){
        requestRecord.setReleaseTime(System.currentTimeMillis());
//        requestRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
//        requestRecord.addFilterInfo(FilterUtils.redisFilterKeyInfo(StringUtils.joinWith("-","filter",requestRecord.getDomain(),"common","queue")));
        requestRecord.setDownload(false);
        requestRecord.setNeedWashPage(true);
        requestRecord.setNeedParsedPage(false);
    }

    public static void main(String[] args) {
        generateVideoCrawlerRecord();
    }

    private static void generateVideoCrawlerRecord(){

        String domainId = "douyin";

        CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId,turnPageItem)
                .httpUrl("/aweme/v1/general/search/single")
                .build();

        itemRecord.setDownload(false);
        HttpPage httpPage = new HttpPage();
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(itemRecord.getHttpRequest().getUrl());
        httpPage.setRequest(httpRequest);
        httpPage.setRawText("");
        itemRecord.setInternalDownloadPage(httpPage);

        itemRecord.tagsCreator().requestTags().getCategoryTag().addKVTag("dy_request_data","");

        System.out.println(JSON.toJSONString(itemRecord));
    }
}
